package sis.ra.founded;

import java.io.File;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;


import sis.ra.partofspeech.wordindex;
import sis.ra.utility.Utils;
import sis.ra.yahoor.result.extractPattern;
import sis.ra.yahoor.result.parseYahooResults;
import sis.ra.yahoor.result.yahooresults;


public class extractYahooResults {

	static int alllines;
	static int compprod;
	
	public static String getYahooResultXML(String query, int resultnum)
	{
		try {
			query = URLEncoder.encode(query, "UTF-8");
		} catch (UnsupportedEncodingException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		String yahooquery="http://boss.yahooapis.com/ysearch/web/v1/"+query+"?format=xml&count="+resultnum+"&filter=-porn&appid=r.n3YhLV34FvfCRDPEJEm83gFUWbioKVrN3kcjeMol6J94rCnQndPPqzuOaIVA--";
		System.out.println(yahooquery);
		String content=Utils.fetchContentfrURL(yahooquery);
		System.out.println(content);
		return content;
	}
	
	public static void getYahooResult()
	{
		String path="C:\\Documents and Settings\\I820753\\Desktop\\dataset\\companylocation\\";
		String file=path+"CompanyLocation.txt";
		String content = Utils.readFile(file);
		String [] t=content.split("\n");
		for (int i=0;i<t.length;i++)
		{
			String line=t[i];
			String companyname=line.split(" ")[0];
			String location=line.split(" ")[1];
			if (line.contains("\" \""))
			{
				companyname=line.split("\" \"")[0].replace("\"", "");
				location=line.split("\" \"")[1].replace("\"", "");
			}
			if (line.contains("\" "))
			{
				companyname=line.split("\" ")[0].replace("\"", "");
				location=line.split("\" ")[1].replace("\"", "");
			}
			if (line.contains(" \""))
			{
				companyname=line.split(" \"")[0].replace("\"", "");
				location=line.split(" \"")[1].replace("\"", "");
			}
			String filename=(path+line).replace("\"", "").replace(",", "").replace("!", "").replace(".", "");
			System.out.println(filename+"\t"+companyname+"\t"+location);
			Utils.writeToFile(path+"companylocation.list", filename+"\t"+companyname+"\t"+location, true, true);
	//		String result=getYahooResultXML(line, 50);
	//		Utils.writeToFile(filename+".yahresult", result, true, true);
		}
	}
	
	public static void extractAbs(String xml, String company, String location)
	{
		String [] lines=xml.split("\n");
		String path="C:\\Documents and Settings\\I820753\\Desktop\\dataset\\companylocation\\";
		String line="";
	
		for (int i=0;i<lines.length;i++)
		{
			if (lines[i].contains("<abstract>"))
			{
				alllines=alllines+1;
			//	System.out.println(company+"\t"+location+"\t"+lines[i]);
				line=lines[i].replace("</b><b>", "");
				line=line.replace(company, "COMPANY");
				line=line.replace(location, "LOCATION");
				line=line.replace(company.replace(".", ""), "COMPANY");
				line=line.toLowerCase().replace(company.toLowerCase(), "COMPANY");
				line=line.toLowerCase().replace(location.toLowerCase(), "LOCATION");
				
				if (line.contains("COMPANY")&&line.contains("LOCATION"))
					compprod=compprod+1;
				else
				{
	//				System.out.println(company+"\t"+location+"\t"+line);
				}
				Utils.writeToFile(path+"companyproduct.abs", company +"\t"+location+"\t"+line, true, true);
			}
		}
	}
	
	public static void matchCompanyLocation()
	{
		String path="C:\\Documents and Settings\\I820753\\Desktop\\dataset\\companylocation\\";
		File [] files=new File(path).listFiles();
		HashMap<String,String> companylocation=getCompanyLocation();
			
		for (int i=0;i<files.length;i++)
		{
			if (!files[i].toString().contains("yahresult")) continue;
			String xml=Utils.readFile(files[i].toString());
			String filename=files[i].toString().replace(".yahresult", "");
			String companyname=companylocation.get(filename).split("\t")[0];
			String location=companylocation.get(filename).split("\t")[1];
		
			extractAbs(xml, companyname, location);
//			ArrayList<yahooresults> rs=parseYahooResults.getYahooResult(xml);
//			for (int j=0;j<rs.size();j++)
//			{
//				System.out.println(rs.get(j).getAbs());
//			}
		}
		System.out.println(alllines+" "+compprod);
	}
	public static HashMap<String,String> getCompanyLocation()
	{
		HashMap<String,String> companyLocation = new HashMap<String,String> ();
		String companynamefile="C:\\Documents and Settings\\I820753\\Desktop\\dataset\\companylocation_training\\companylocation.list";
		String [] companylocation=Utils.readFile(companynamefile).split("\n");
		for (int i=0;i<companylocation.length;i++)
		{
			if (companylocation[i].length()<2) continue;
	//		System.out.println(companylocation[i]);
			String[] t=companylocation[i].split("\t");
			String filename=t[0];
			companyLocation.put(filename, t[1]+"\t"+t[2]);
		}
		return companyLocation;
	}
	
	public static void extractVerbPattern()
	{
		String path="C:\\Documents and Settings\\I820753\\Desktop\\dataset\\companylocation_training\\companyproduct.abs";
		String [] lines=Utils.readFile(path).split("\n");
		int countline=0;
		int verbcount=0;
		for (int i=0;i<lines.length;i++)
		{
			System.out.println(i+"\t"+lines[i]);
			if (lines[i].contains("company")&&lines[i].contains("location"))
			{
				if (lines[i].split("\t").length<3) continue;
				String line=lines[i].split("\t")[2];
				line=line.trim().replace("<abstract><![cdata[","").replace("]]></abstract>", "");
				String [] parts=line.split("<b>...</b>");
				
				for (int j=0;j<parts.length;j++)
				{	
					if (parts[j].trim().length()>2&&parts[j].contains("company")&&parts[j].contains("location"))
					{
						String thisline=parts[j].replace("<b>company</b>", "COMPANY").replace("<b>location</b>", "LOCATION");
						if (thisline.contains("COMPANY")&&thisline.contains("LOCATION"))
						{
							countline=countline+1;
							HashMap<Integer,wordindex> posresult=parseYahooResults.pos(thisline.trim(), "text");
							ArrayList<String> resultlist=parseYahooResults.getResultList (posresult);
							String result="";
							String verb="";
							String thisverb="";
							int companyverbdist=0;
							int verblocationdist=1000;
							int companypos=thisline.trim().indexOf("COMPANY");
							int locationpos=thisline.trim().indexOf("LOCATION");
							if (companypos>locationpos)
								companyverbdist=thisline.substring(locationpos, companypos).split(" ").length;
							else
								companyverbdist=thisline.substring(companypos,locationpos).split(" ").length;
							for (int n=0;n<resultlist.size()-1;n++)
							{
								if (resultlist.get(n).contains("V-")&&!resultlist.get(n+1).contains("V-"))
								{
									String thverb=resultlist.get(n).trim().split("\t")[1];
									System.out.println(thverb);
									int verbpos=thisline.trim().indexOf(thverb);
									if ((verbpos>companypos&&verbpos>locationpos)||(verbpos<companypos&&verbpos<locationpos)) continue;
									int thisverblocationdist=0;
									if (verbpos>companypos&&verbpos<locationpos)
									{
										thisverblocationdist=thisline.substring(verbpos, locationpos).split(" ").length;
									}else
									{
										thisverblocationdist=thisline.substring(locationpos, verbpos).split(" ").length;
									}
						//			verb=verb+"\t"+resultlist.get(n).trim();
									if (verblocationdist>thisverblocationdist)
									{
										verblocationdist=thisverblocationdist;
										thisverb=thverb;
									}
								}
								result=result+resultlist.get(n).trim();
							}
							if (resultlist.get(resultlist.size()-1).contains("V-"))
							{
								String thverb=resultlist.get(resultlist.size()-1).trim().split("\t")[1];
								if (thisline.indexOf(thverb)<thisline.trim().indexOf("LOCATION")) continue;
								int thisverblocationdist=thisline.substring(locationpos, thisline.trim().indexOf(thverb)).split(" ").length;
								if (verblocationdist>thisverblocationdist)
								{
									verblocationdist=thisverblocationdist;
									thisverb=thverb;
								}
								verbcount=verbcount+1;
							}
							result=result+resultlist.get(resultlist.size()-1).trim();
							if (thisverb.length()<2) continue;
							String stemverb=extractPattern.stem(thisverb);
							System.out.println(companyverbdist+"\t"+verblocationdist+"\t"+thisverb+"\t"+thisline.trim());
							Utils.writeToFile("C:\\Documents and Settings\\I820753\\Desktop\\dataset\\companylocation\\verbs.sent", companyverbdist+"\t"+verblocationdist+"\t"+thisverb+"\t"+stemverb+"\t"+thisline.trim(), true, true);
							verbcount=verbcount+1;
							
					//		System.out.println(result);
							if (verb.length()>2)
								System.out.println(verb.trim());
						}
					}					
				}
			}
		}
		System.out.println(countline+" "+verbcount);
	}
	public static void main(String[] args) {
//		getYahooResult();
//		matchCompanyLocation();
//		extractVerbPattern();
		
		String path="C:\\Documents and Settings\\I820753\\Desktop\\dataset\\companylocation_training\\verbs.sent";
		String [] lines=Utils.readFile(path).split("\n");
		HashMap<String, Integer> avgcompanylocation=new HashMap<String, Integer>();
		HashMap<String, Integer> avgverblocation=new HashMap<String, Integer>();
		HashMap<String, Integer> verbfreq=new HashMap<String, Integer>();
		for (int i=0;i<lines.length;i++)
		{
			if (lines[i].length()<2) continue;
			String [] t=lines[i].split("\t");
			Integer companylocation=Integer.parseInt(t[0]);
			Integer locationverb=Integer.parseInt(t[1]);
			String verb=t[3];
			if (verbfreq.containsKey(verb))
			{
				Integer verbf=verbfreq.get(verb)+1;
				verbfreq.remove(verb);
				verbfreq.put(verb, verbf);

				Integer vl=avgverblocation.get(verb)+locationverb;
				avgverblocation.remove(verb);
				avgverblocation.put(verb, vl);
				Integer cp=avgcompanylocation.get(verb)+companylocation;
				avgcompanylocation.remove(verb);
				avgcompanylocation.put(verb,cp);
			}else
			{
				verbfreq.put(verb, 1);
				
				avgverblocation.put(verb, locationverb);
				avgcompanylocation.put(verb,companylocation);
			}
		}
		
		Iterator<String> it=verbfreq.keySet().iterator();
		while (it.hasNext())
		{
			String verb=it.next();
			Integer freq=verbfreq.get(verb);
			double avgcomploc=(double)avgcompanylocation.get(verb)/(double)freq;
			double avgverbloc=(double)avgverblocation.get(verb)/(double)freq;
			String outline=verb+"\t"+freq+"\t"+avgcomploc+"\t"+avgverbloc+"\t"+(double)freq/lines.length+"\t"+(double)1/avgcomploc+"\t"+(double)1/avgverbloc;
			System.out.println(outline);
		}
	}

}
